package com.spider.ask120.magic.WebMagic;

import java.util.List;

import org.apache.http.HttpHost;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.handler.SubPageProcessor;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 健康资讯 频道
 * @author Jxw
 *
 */
public class Zixun  implements PageProcessor{
	static String jkzxUrl = "http://m.120ask.com/news/";
	static String huatiList = "http://m.120ask.com/news/s/\\w+";
	static String huatiPage = "http://m.120ask.com/news/s/\\w+/\\?p=[0-9]+";
	static String viewRegex = "http://m.120ask.com/news/p/[0-9]+";
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() { 
		return site;
	}
		
	// process是定制爬虫逻辑的核心接口
	
	public void process(Page page) { 
		//进入频道页
		if(page.getUrl().regex(jkzxUrl+"$").match()) { 
			//获取到话题页列表URL 
			List<String> huati = page.getHtml().links().regex(huatiList).all(); 
			page.addTargetRequests(huati);
		
		//进入话题页
		}else if(page.getUrl().regex(huatiList+"\\?{0}$").match()){
			//遍历话题分页URL，及每页文章  
			String htPage = page.getUrl()+"/?p=1";  
			page.addTargetRequest(htPage);
			
		//进入分页
		}else if(page.getUrl().regex(huatiPage).match()){ 
			List<String> viewPage = page.getHtml().links().regex(viewRegex).all(); 
			for(String view : viewPage){
				System.out.println(view);
				System.out.println("id : " + view.substring(view.indexOf("/p/")+3)); 
				System.out.println("image : " + page.getHtml().xpath("//a[@href='"+view+"']/dt/html()")); 
				System.out.println("title : " + page.getHtml().xpath("//a[@href='"+view+"']/dd/h3/html()")); 
				System.out.println("abstract : " + page.getHtml().xpath("//a[@href='"+view+"']/dd[2]/html()")); 
				page.addTargetRequest(view);
			}
			
			//判断下一页URL是否存在 
			String path = page.getUrl().toString();
			String pageIndex = path.substring(path.indexOf("p=")+2); 
			String nextPage = path.substring(0, path.indexOf("p=")+2) + (Integer.parseInt(pageIndex)+1);
			if(page.getHtml().links().regex("http://m.120ask.com/news/s/\\w+/\\?p=["+pageIndex+"]").match()){
				//判断当前页 是否有文章列表
				if(page.getHtml().$(".list_box dl").match()){
					page.addTargetRequest(nextPage);
				}
			}
			
			
		//进入详情页
		}else if(page.getUrl().regex(viewRegex).match()){  
			page.putField("title", page.getHtml().xpath("//div[@class='pr']/h2/text()")); 
			page.putField("create_time", page.getHtml().$(".s_p span","text"));
			page.putField("source", page.getHtml().$(".s_p span:nth-child(2)","text"));
			page.putField("click", page.getHtml().$("#r_num","text"));
			page.putField("praise", page.getHtml().$("#g_num","text"));
			page.putField("pageContent", page.getHtml().xpath("//div[@class='s_content']/html()")); 
		}
		 
		
	}
  
	public static void main(String[] args) {
		System.setProperty("http.maxRedirects", "50");  
        System.getProperties().setProperty("proxySet", "true");  
        // 如果不设置，只要代理IP和代理端口正确,此项不设置也可以  
        String ip = "93.91.200.146";  
        ip = "221.130.18.5";  
        ip = "221.130.23.135";  
        ip = "221.130.18.78";  
        ip = "221.130.23.134";  
        ip = "221.130.18.49";  
        ip = "111.1.32.36";  
        ip = "221.130.18.49";  
        ip = "221.130.18.49";  
        System.getProperties().setProperty("http.proxyHost", ip);  
        System.getProperties().setProperty("http.proxyPort", "80");
//		Spider.create(new JkzxAction())
//			.addUrl(jkzxUrl).thread(1).run();
		Spider.create(new Zixun())
		.addUrl("http://m.120ask.com/news/s/spaq/?p=1").thread(1).run();
		
        
 
    }
 
}
